import json
import selectors
import time

import scrapy
import re
import pypinyin


class FirstSpider(scrapy.Spider):
    name = 'first_spider'

    def start_requests(self):
        search_key = ['静夜思']
        for key in search_key:
            url = 'https://www.shicile.com/s?wd=' + key
            yield scrapy.Request(url=url, headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"},
                                 callback=self.search)

    def search(self, response, **kwargs):
        tr = response.xpath("//table//tr")
        index = 0
        for tr_item in tr:
            if index == 0:
                pass
            else:
                a = tr_item.xpath("./td[2]/a/@href").get()
                a = 'https://www.shicile.com' + a
                yield scrapy.Request(url=a, headers={
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"},
                                     callback=self.parse)
            index = index + 1
            time.sleep(2)

    def parse(self, response, **kwargs):
        data = {}
        title = response.css(".card-title ruby").get()
        title = re.sub("</ruby>", "", title)
        title = re.sub("<ruby>", "", title)
        title = re.sub("\s*<rt>(.*?)</rt>\s*", "", title)
        author = response.css("span+ span ruby").get()
        author = re.sub("</ruby>", "", author)
        author = re.sub("<ruby>", "", author)
        author = re.sub("\s*<rt>(.*?)</rt>\s*", "", author)
        title = title.replace(" ", "")
        title =  title.replace('<rt class="symbol"> </rt>',"");
        author = author.replace(" ", "")
        data['book'] = title
        data['author'] = author
        all_line = []
        ##.py-content ruby
        if (len(response.xpath("//ruby/ruby").getall()) > 0):
            tag = "//ruby/ruby"
        elif (len(response.xpath("//p[@class='mid-compact py-content']/ruby").getall()) > 0):
            tag = "//p[@class='mid-compact py-content']/ruby"
        for item in response.xpath(tag).getall():
            str = item.replace("</ruby>", "")
            str = str.replace("<ruby>", "")
            match_objs = re.findall(r"(.*?)<rt>(.*?)</rt>", str)
            line = []
            for match in match_objs:
                txt = match[0].replace("\xa0", "")
                txt = txt.replace(" ", "")
                pinyin = match[1]
                pinyin = pinyin.replace(" ", "")
                if len(pinyin) == 0:
                    pinyin_arr = pypinyin.pinyin(txt)
                    if len(pinyin_arr) != 0:
                        pinyin = pinyin_arr[0][0]
                line.append(
                    {
                        "py": pinyin,
                        "txt": txt
                    }
                )
            all_line.append(line)

        data['list'] = all_line
        file_name = data['book'] + "_" + data['author'] + ".json"
        with open("./my/" + file_name, mode="w", encoding="utf-8") as fp:
            json.dump(data, fp, ensure_ascii=False)
        yield data